Conversation
|
Important Review skippedToo many files! This PR contains 1475 files, which is 1375 over the limit of 100. To get a review, narrow the scope: Upgrade to a paid plan to raise the limit. Usage-priced reviews support at most 300 files. ⚙️ Run configurationConfiguration used: Organization UI Review profile: CHILL Plan: Pro Run ID: ⛔ Files ignored due to path filters (10)
📒 Files selected for processing (1565)
You can disable this status message by setting the Warning Billing warning: we have not been able to collect payment for this subscription for more than 72 hours. Please update the payment method or pay any pending invoices in Billing to avoid service interruption. |
![github-advanced-security[bot]](https://avatars.githubusercontent.com/in/57789?s=60&v=4){kind=small}
| } | ||
| store.mu.Lock() | ||
| defer store.mu.Unlock() | ||
| return scanAgentRun(store.connection.QueryRow(runSelect+" WHERE id = ?", id), "Run") |
| } | ||
| store.mu.Lock() | ||
| defer store.mu.Unlock() | ||
| runResult := scanAgentRun(store.connection.QueryRow(runSelect+" WHERE id = ?", runID), "Continuation") |
| } | ||
|
|
||
| func queryAgentLogs(connection *sql.DB, suffix string, arguments ...any) ([]work.LogChunk, error) { | ||
| rows, err := connection.Query("SELECT run_id, sequence, stream, text, created_at FROM agent_log_chunks "+suffix, arguments...) |
| pli.astype(np.float32).tofile(os.path.join(OUT, "pli.f32")) | ||
| layer_out.tofile(os.path.join(OUT, "layer_out.f32")) | ||
| attn_res.tofile(os.path.join(OUT, "attn_res.f32")) | ||
| json.dump({"ids": IDS, "T": T, "H": H, "NL": NL, "PLID": PLID}, open(os.path.join(OUT, "manifest.json"), "w")) |
| layer_out[li] = h | ||
| print(f"layer {li:2d} {'S' if is_sliding else 'G'}{'' if has_kv else ' shared->' + str(prev[li])}: |h| max {np.abs(h).max():.3f}") | ||
|
|
||
| os.makedirs(OUT, exist_ok=True) |
| # SDPA, scale 1.0, causal (+ sliding window), GQA broadcast kv head | ||
| o = np.zeros((T, NH, hd), dtype=np.float64) | ||
| gqa = NH // nkv | ||
| for hh in range(NH): |
| attn_res = np.zeros((NL, T, H), dtype=np.float32) | ||
| kv_bank = {} | ||
|
|
||
| for li in range(NL): |
| by_type = {} | ||
| for i in range(M): | ||
| by_type[LTYPES[i]] = i | ||
| for j in range(M, NL): |
| } | ||
| s.mu.Lock() | ||
| defer s.mu.Unlock() | ||
| item, err := scanItem(s.conn().QueryRow(itemSelect+" WHERE id = ?", id)) |
| if count == 0 { | ||
| return core.Fail(datasetNotFoundErr("tui.duckDatasetStore.DatasetArchive", "dataset", id)) | ||
| } | ||
| d, err := scanDataset(s.conn().QueryRow(datasetSelect+" WHERE id = ?", id)) |
| } | ||
| s.mu.Lock() | ||
| defer s.mu.Unlock() | ||
| d, err := scanDataset(s.conn().QueryRow(datasetSelect+" WHERE slug = ?", slug)) |
| } | ||
| s.mu.Lock() | ||
| defer s.mu.Unlock() | ||
| d, err := scanDataset(s.conn().QueryRow(datasetSelect+" WHERE id = ?", id)) |
| // can observe or claim the same seq value. | ||
| func (s *duckDatasetStore) nextSeqLocked(table string) (int64, error) { | ||
| var next int64 | ||
| if err := s.conn().QueryRow("SELECT COALESCE(MAX(seq), 0) + 1 FROM " + table).Scan(&next); err != nil { |
| if count == 0 { | ||
| return core.Fail(datasetNotFoundErr("tui.duckDatasetStore.ItemArchive", "item", id)) | ||
| } | ||
| item, err := scanItem(s.conn().QueryRow(itemSelect+" WHERE id = ?", id)) |
| "top5_ids": [int(i) for i in np.argsort(-logits)[:5]], | ||
| "top5_vals": [float(logits[i]) for i in np.argsort(-logits)[:5]], | ||
| } | ||
| with open(out_path, "w") as f: |
| v_first = None | ||
| state = [{"wkv": None, "shift1": None, "shift2": None} for _ in range(num_layers)] | ||
| captured = {} | ||
| for li in range(num_layers): |
|
|
||
| x = embed[prompt_ids] | ||
| v_first = None | ||
| state = [{"wkv": None, "shift1": None, "shift2": None} for _ in range(num_layers)] |
| norm_b = st.get("model.norm.bias") if st.has("model.norm.bias") else None | ||
| lm_head = st.get("lm_head.weight") if st.has("lm_head.weight") else embed | ||
|
|
||
| layers = [load_layer(st, li) for li in range(num_layers)] |
| ckpt_dir = sys.argv[1] | ||
| out_path = sys.argv[2] if len(sys.argv) > 2 else "oracle_fixture.json" | ||
|
|
||
| with open(ckpt_dir + "/config.json") as f: |
|
|
||
| class SafeTensors: | ||
| def __init__(self, path): | ||
| self.f = open(path, "rb") |
DecodeLogitsStep replaces GreedyDecode/DetectLanguage's per-step full- sequence recompute: per decoder layer, a SelfAttnCache grows by one batch's K/V per step (mhaCore gains an offset parameter so a cached causal pass reproduces a whole-sequence recompute bit-for-bit — see attention.go/decoder.go doc comments for the row-wise-independence argument). Cross-attention Q is now projected for only the new row(s) too. DecodeLogits (full recompute) is kept unchanged as the parity oracle: TestDecodeLogitsStep_Good proves bit-identical logits across fresh-single-token, multi-token-prefill, and cache-continuation call shapes. Receipt (live whisper-tiny, hello16k.wav, same 10-token transcript both sides): decode loop 1.508s -> 0.385s (~3.9x, prefill+9 steps). End-to-end Transcribe barely moves (~22.3s -> ~20.2s) because the encoder's O(1500^2) self-attention dominates wall-clock (~18s, ~88% of total, run-to-run noise alone ~0.9s) -- out of this lane's scope per the design's phasing. The originally-cited "~23s" was mostly encoder time, not decode recompute. Co-Authored-By: Virgil <virgil@lethean.io>
The shipped MLX metallib only instantiates sdpa_vector for a fixed class of head dims (64/96/128/256, one C++ template per width); head dim 32 has no pipeline anywhere, and every call site's absent-width behaviour differed (some errored immediately, some fell back to a different kernel family that also lacked the width) — the shape train_trainer_layers_test.go / train_trainer_mask_test.go moved to head dim 64 specifically to dodge, per their own comments. Adds kernels/lthn_sdpa_rtdim.metal: MLX's sdpa_vector single-pass loop (the same body lthn_sdpa_vector_q8.metal already ports for int8 K/V) with head_dim taken as a RUNTIME buffer parameter instead of a template argument — one compiled kernel serves every absent width (a positive multiple of the 32-lane simdgroup width, up to the 256 register-array cap), not just 32. Plus its 2pass_1 sibling, proven correct in isolation (partials/sums/maxs byte-band-match the fixed kernel's own output at head_dim=64) but NOT wired end to end: pass 2 (the merge kernel) is ALSO a per-headDim fixed-only pipeline with no runtime-dim port, so wiring pass 1 alone would resolve a pipeline and still fail at pass 2 — a named, honest remaining boundary rather than a rushed, unvalidated pass-2 written without MLX's source to check against. sdpa_rtdim.go's sdpaVectorDispatchForHeadDim is the single chokepoint every non-sinks, non-q8 single-pass call site now resolves through: the fixed per-headDim pipeline when the metallib carries it (byte-identical, untouched), or the fallback logged once per distinct absent width — never an error, never a different fallback per site. Wired into SDPAInto, encSDPAStrided, encSDPADecodeAt (the central per-token decode chokepoint), attentionBlockInto, and layer.go's decodeLayerInto. encSDPADecodeAt's 2-pass branch now falls all the way through to the single-pass fallback when either fixed 2-pass pipeline is unavailable, so decode at an absent head dim has no cliff at the 2-pass knee (kvLen>=1024) — correct at any length, just without the 2-pass long-context parallelism past it. Scope: the ICB record-once/replay decode path (icb.go, decode_forward_arch_icb.go, icb_layer.go, decode_forward_icb.go) has its OWN separate SDPA pipeline resolver with no fallback — discovered via TestSdpaRTDim_TrainerSessionAtHeadDim32_Good (OpenSession at head_dim=32 fails there even with LTHN_DECODE_ICB=0, which only gates ICB use, not the unconditional record-if-eligible at session construction). Confirmed materially larger (4+ call sites, record/ replay semantics) and left as a named follow-up; the test forces the non-ICB trace path via LTHN_NATIVE_TRACE=1 to prove the fix through the real per-token decode chokepoint instead. Gates: kernel vs host float reference at hd=32 and vs the fixed kernel at hd=64 (cosine=1.0, worst|d|=0 across kvLen 1/17/32/65/257, spanning the BN=32 sweep boundary) — TestSdpaRTDim_SDPA32_Good, TestSdpaRTDim_MatchesFixedAtHeadDim64_Good. #28 repro: a synthetic ArchSession forward at head_dim=32 (ForwardCaptureHiddens, the same per-token StepWithID chokepoint every generate() call uses) — no error. No cliff past the 2-pass knee at head_dim=32 — TestEncSDPADecodeAt_HeadDim32PastTheKnee_Good. task metallib:kernels + task build clean; 115 sdpa-family tests green with MLX_METALLIB_PATH set, including the full pre-existing AttentionBlock/DecodeLayer/ EncSDPADecodeAt regression set (two pre-existing, unrelated TestAttentionBlockICB*AllocationBudget failures confirmed present on unmodified dev — icb.go untouched by this change). Co-Authored-By: Virgil <virgil@lethean.io>
…36 tail) Two defects closed the standalone-recurrent serve seam: 1. metalBackend.LoadModel's post-load switch served only *NativeTokenModel and *composed.ComposedTokenModel; any other model.TokenModel (rwkv7's, mamba2's) hit the default arm's "unservable" error. Generalised the composed-only bridge into a third arm keyed on the neutral model.SessionModel interface — composedTextModel is renamed sessionTextModel (the old name became a lie once it also serves standalone-recurrent archs) and now declares ChatML for Qwen model_types (unchanged), a new plain-completion dialect for rwkv7/mamba2 (no bracket markers neither checkpoint was tuned on), and the gemma fallback otherwise. composedEngineSession is reused verbatim — its shape was already arch-neutral. 2. tokenizer.LoadTokenizer(tokenizer.json) is hardcoded and fails for rwkv7 checkpoints (vocab .txt + custom code, no tokenizer.json). Chose option (a): WorldTokenizer grows DecodeToken/DecodeOne/TokenID/EOS to satisfy engine.TextTokenizer BY SHAPE, no import of engine (AX-8 preserved). The World vocab is a fixed artefact shared by the whole checkpoint family, so it is embedded in the binary (data/rwkv_vocab_v20230424.hex, moved out of testdata/ since it is now production data) rather than expected on disk — a real HF snapshot ships only the unparseable Python-repr .txt. loadServeTokenizer implements the fallback: tokenizer.json when present (mamba2, byte-identical), else the arch tokenizer for *rwkv7.RWKV7TokenModel, else the original error. Live-caught a THIRD, pre-existing bug while proving the seam: engine.TextModel.chatTemplate() used chatTmpl.Open == "" as its "was a template ever resolved" sentinel, indistinguishable from a real declaration whose Open genuinely is "" (the new plain-completion dialect) — silently reverting to gemma's <start_of_turn> markers, which an RWKV7 checkpoint tokenises as junk and degrades into confused, prompt-echoing continuations. Fixed with an explicit chatTmplSet flag; regression test added. Also added NumLayers/HiddenSize to RWKV7TokenModel/MambaTokenModel (mirroring composed.ComposedTokenModel) so ModelInfo reports real geometry through the generic arm, and fixed examples/pkg/rwkv7-generate's now-stale vocab path (testdata -> data) to use the embedded tokenizer by default. Live receipts (RWKV7-Goose-World2.8-0.1B-HF): - examples/pkg/generate (raw TextModel.Generate, no template): "The capital of France is Paris.\n- The capital of the United States is Washington, D.C.\n- The" — matches the library-level receipt's structure exactly. - lem generate -temp 0 / lem serve + curl (both Chat-framed via the new plain-completion dialect — lem generate always chat-frames despite its name): "A: Paris\n\nQ: ..." — correct answer, honestly non-fabricated framing. - mamba2 regression: model/arch/mamba2 (70 tests) + engine/metal's mamba2 tests all green. Co-Authored-By: Virgil <virgil@lethean.io>
Absent head-dim widths now route to lthn_sdpa_vector_rtdim_bf16 (+ its 2pass_1 sibling, written and proven but unwired pending a pass-2 port — the 2-pass branch falls through to single-pass, correct at any length) through one dispatch chokepoint with a one-time notice. Byte-exact parity: hd32 vs host reference AND forced-rtdim vs the fixed hd64 kernel, cosine 1.000000 / worst|d|=0 across kvLen 1..257; pass-1 within the sinks byte-band at kvLen=2000. Named boundary: the ICB record path has its own resolver family with no fallback (pre-existing, refuses at construction — follow-up, not attempted blind). Co-Authored-By: Virgil <virgil@lethean.io>
…ne/rwkvserve) The register switch generalises over model.SessionModel (composedTextModel renamed sessionTextModel — the name stopped being true); tokenizer fallback tokenizer.json → embedded World vocab (rwkv7) → honest error; plain-completion dialect for the standalone recurrents. Third bug found live: chatTemplate's empty-Open sentinel silently reverted undeclared-marker dialects to gemma markers — fixed with an explicit set-flag + regression. Live: serve+curl answers Paris; mamba2 70 tests green. Co-Authored-By: Virgil <virgil@lethean.io>
Transcriber capability interface (go/inference.go, the VisionModel/ AudioModel stability-contract pattern) discovered by type-assertion; whisper.Model.TranscribeAudio adapts Transcribe's Options/Result shape to the primitive-string contract so the whisper package satisfies it structurally, with no import in either direction. openai.TranscriptionHandler serves OpenAI-compatible multipart upload plus this engine's JSON+base64 data-URL extension (the image_url precedent -- data: URLs only, no remote fetch), response_format json/text, always mounted so a non-whisper (or model-less) serve gets the clean 400 capability refusal rather than a 404. Loader integration: whisper.IsWhisperCheckpoint cheaply probes --model's config.json; RunServe's detectAndLoadWhisper runs before the ordinary chat hot-swap/multi-model wiring and routes the WHOLE process into ASR-only serving when it matches (fatal-at-boot on a load failure, matching the outbound-policy/embed-model fail-closed precedent) -- the v1 "one process, one purpose" shape: chat/embeddings degrade to the existing model-less refusal, exactly as a plain --model "" start already behaves. Gates: compat-shape + capability-refusal tests in serving/provider/openai/transcription_test.go and serving/compat/admin_test.go; mux/serve wiring re-verified against the existing suite (2620 tests, serving tree + whisper package, all green). Live round trip (task build; bin/lem serve --model <whisper-tiny>; curl both multipart and JSON data-URL) returns the exact golden transcript byte-for-byte in ~21s (encoder-dominated, see the KV-cache commit) via both surfaces; wrong-method/missing-file/ remote-URL-refused/chat-route-model-less all verified live too. Co-Authored-By: Virgil <virgil@lethean.io>
/v1/audio/transcriptions (multipart + base64 data-URL, image precedent; always mounted, 400 without a transcriber; remote URLs refused) via the Transcriber stability-contract interface; serve routes a whisper --model into ASR-only serving fail-closed. Decoder self-attn KV cache bit-exact vs the recompute oracle (maxAbsDiff 0); decode loop 1.508s -> 0.385s. Honest receipt: end-to-end ~20s because the encoder's O(1500^2) self-attn dominates (~88%) — the named next perf target, not claimed fixed. Live: exact golden transcript through both curl surfaces; chat on a whisper serve degrades to the model-less refusal. Co-Authored-By: Virgil <virgil@lethean.io>
…llback (#48) The TQ live-KV v1 sent prompt prefill down the per-token replay: a ~65x prefill regression versus the batched pass, disqualifying for the very long-context case TQ exists for. This wires the batched/chunked dense prefill to be TQ-aware. Store side: the chunk's roped/normed K/V land into the code caches via the existing lthn_tq_kv_store kernel dispatched at grid width kvHeads*numRows (encTQKVStoreRows) — one dispatch per chunk, no new store kernel. Read side: a NEW lthn_tq_kv_dequant_bf16 kernel reconstructs the prior history codes (Piᵀ*gamma*centroid) into a reusable per-layer bf16 scratch and the ordinary chunk-attention SDPA (multiQ/GEMM/flash) scores against it unchanged. The scratch is INCREMENTAL across chunks (a high-water mark appends only each chunk's own rows — never re-dequantising the history, the O(n²) trap) and is freed at the prefill->decode transition, so the deep-context residency win is kept during generation. Root-caused a hard bug the batched path exposed: lthn_tq_kv_store read red[0] (gamma) then reused red for u WITHOUT a barrier. At head dim 512 (16 simd-groups) a group reaching the overwrite raced a lagging group still reading gamma, handing it u[0] — benign at 1 threadgroup (the per-token decode kept groups in step) but a hard corruption once the batched prefill interleaves many threadgroups. Added the barrier; the store→dequant round-trip drops from >100% error to the ~0.1 codec floor at every instantiated head dim. Receipts (real e2b-4bit, this box — machine load-caveated, a 99% peer process ran during some runs): - retrieval: batched RETRIEVES "7412" through the code caches in ~1.5s vs ~11.4s per-token (LTHN_TQ_BATCHED_PREFILL=0); the fact ~1047 tokens back surfaces. - decode live gate unchanged: top-1 62/64, max |logit Δ| 6.16. - prefill throughput (3482-tok prompt): batched 7303 tok/s vs per-token 113 tok/s (~65x). Stable across reruns. - parity: batched-vs-sequential next-token argmax AGREES, max |logit Δ| 0.125 (codec band, not byte-tight — different accumulation order + bf16 reconstruction rounding; stated in the test). LTHN_TQ_BATCHED_PREFILL=0 pins the pre-#48 per-token path (the A/B lever). Sliding/local layers, non-TQ layers and the below-fold-gate small-chunk case are byte-identical untouched (decline to the per-token replay). Co-Authored-By: Virgil <virgil@lethean.io>
Store lands whole chunks in one dispatch; NEW lthn_tq_kv_dequant_bf16 reconstructs prior history into a per-layer scratch with an incremental high-water mark (never re-dequantising), freed at prefill->decode so the residency win survives generation. LTHN_TQ_BATCHED_PREFILL=0 is the A/B lever. Load-bearing race FOUND+FIXED in lthn_tq_kv_store: gamma read and red[] reuse had no barrier — benign per-token, corrupt under batched interleave at hd512. Receipts: 3482-token prompt 7303 vs 113 tok/s; retrieval prefill 11.40s -> 1.49s; parity argmax-agree (codec band 0.125); TQ suite 80 green; live gates unchanged (62/64). Co-Authored-By: Virgil <virgil@lethean.io> # Conflicts: # cli/lthn_kernels.metallib.gz
…ned the suite sdpaVectorDispatchForHeadDim latched a width as missing on ANY fixed- lookup failure and the rtdim PSO getters latched their errors — one transient library-teardown window in a full-suite run rerouted every later caller and cascaded 199 failures. Fixed lookup now runs every call (PSO success caches make it cheap), the missing-map gates only the one-time notice, and both rtdim getters retry until success. Co-Authored-By: Virgil <virgil@lethean.io>
An absent fixed width with a live customLibrary now SERVES via the runtime-dim fallback (pinned), and the kernel-not-found error leg needs both libraries gone (custom nulled within the wrong-main scope). This test's mid-suite library swap was the transient window that the old error-latching resolvers turned into the 199-failure cascade. Co-Authored-By: Virgil <virgil@lethean.io>
…ches registered in the reset, inline null/restore, dead err vars dropped The nulled-custom error leg needs the rtdim PSO caches actually cleared (they were unknown to resetNativePipelineCachesForCoverage) and the restore must happen before the success half (t.Cleanup fires at test end). The fallback contract is now pinned both ways: absent width + live custom serves; both libraries gone errors. Co-Authored-By: Virgil <virgil@lethean.io>
… >15-token divergence is the serve lane's chat framing + bf16-head seam (#36)
Adds the mamba2-pair parity test family for rwkv7 (TestRWKV7DeviceProjMatchesReference
at every real 0.1B-checkpoint GEMM shape; a real-checkpoint device-vs-host band over 32
sequential carried-state steps; a closed-loop 32-token greedy verdict) plus
RWKV7Session.HeadLogits, an exported f32 seam mirroring composed.HeadLogitsHost for
device-vs-host receipts from outside the package.
Verdict: outcome (c). rwkv7.ProjMatMul/ProjMatMulInto (the steel GEMM hook) produces
BYTE-IDENTICAL greedy output to the pure-host path over 32 closed-loop tokens on the
real RWKV7-Goose-World2.8-0.1B-HF checkpoint (band test: logit cosine 1.00000000,
zero own-argmax disagreements) — the device hook is fully exonerated. The reported
divergence traces to two serve-lane factors, neither the device hook: (1) bin/lem
generate's stateless path always calls TextModel.Chat, which frames the prompt through
rwkv7's declared plain-completion template ("\n"+content+"\n\n") — a different token
stream from position 0, proven by direct tokenisation comparison; (2) the
SessionModel/TokenModel serve contract's bf16-rounded LM head (vs RWKV7Session's own
f32 argmax) independently diverges from the pure-host reference at generated position
22 on an identical raw prompt — closely matching the reported "~15 tokens" onset.
New test files live in engine/metal rather than model/arch/rwkv7: that package's test
binary is shared across ALL its _test.go files, and importing engine/metal from there
would run native's init() for the whole suite, silently switching every host-only rwkv7
test onto the Metal GPU.
Gates: go build ./... and go vet ./engine/metal/ ./model/arch/rwkv7/ clean; all new
tests green (MLX_METALLIB_PATH set, scoped -run); the full rwkv7 package suite green
(122 tests); TestRWKV7RealCheckpointOracle and TestRWKV7RealCheckpointGenerationSmoke
green (RWKV7_SMOKE_DIR); the full engine/metal suite shows zero rwkv7/mamba2 failures
(pre-existing SDPA-pipeline failures elsewhere in that run trace to a stale borrowed
metallib build artefact, unrelated to this change — verified by diverged kernel-source
git history, not a regression).
Co-Authored-By: Virgil <virgil@lethean.io>
Steel-GEMM hook byte-identical to pure-host over 32 closed-loop greedy steps on the real Goose 0.1B (logit cosine 1.00000000, zero argmax disagreements; 18-shape proj parity worst |d| ~3e-5). The reported divergence decomposed: lem generate always Chat-frames (different stream from position 0, by design), and the serve contract's bf16 LM-head rounding flips a near-tie argmax at generated position 22 vs the session's f32 argmax — documented, not a defect. HeadLogits f32 seam exported for receipts. Co-Authored-By: Virgil <virgil@lethean.io>
…tion (#48)
Single-pass TQ decode (lthn_sdpa_vector_tq, kernels/lthn_tq_kv.metal) now
folds Πq and Πᵀy into itself — one threadgroup per head already has the
whole row in threadgroup memory, so each fold runs exactly once, same
total FLOPs, just no separate lthn_tq_rot_rows_bf16 dispatch either side.
q reads RAW, out receives the FINAL (unrotated) value directly — the same
shape as the plain/q8 SDPA ops. Buffer 15 (free on the single-pass ABI)
carries the resident Π the fold reads.
The 2-pass pair (lthn_sdpa_vector_2pass_1_tq) stays UNFUSED and unchanged:
its pass-1 grid runs `blocks` threadgroups per kv-head (up to 512), so
folding an O(d²) rotation there would redo that work once per block
instead of once per head — the house fusion rule the opposite way. Its
ABI was also already at 16/16 binds. Pass-2 (MLX's sdpa_vector_2pass_2,
which owns the merged output vector) is shared kernel code outside this
lane's ownership. The recorder (decode_forward_arch_icb.go) now branches
on sdpa2PassICBBlocks: single-pass emits ONE op per TQ-reading layer per
token instead of three (rotate + SDPA + unrotate -> SDPA alone), and the
qRotTQ/attnRotTQ staging buffers are no longer allocated for that lane.
tq_unpack_idx (both SDPA kernels + the batched-prefill dequant kernel)
widens from a per-bit loop to a masked 16-bit word read — bit-identical
output, fewer device reads. lthn_turboquant.metal's S2 dequant kernel gets
the same widening for consistency.
Gates (scoped, this box, single-attempt local wall-clock — not a
dedicated bench rig, no other lane running):
- MLX_METALLIB_PATH=... go test -run 'TQ|TurboQuant' ./engine/metal/:
62 passed, 0 failed, 3 skipped (LEM_REAL_E2B-gated). Single-pass
parity band tightened to <=0.0020 (was <=0.0041 typical/0.0369 worst)
— the fusion also removes 2 intermediate bf16 round-trips. 2-pass band
unchanged (<=0.0008), as expected for an unfused kernel.
- LEM_REAL_E2B=1 go test -run TestRealE2BTurboQuantLive ./engine/metal/:
both PASS — top-1 agreement 62/64 (max |logit delta| 6.16, floor 12),
retrieval fact recovered at ~1047 tokens back. maxLen=2048 exercises
the (unchanged) 2-pass lane.
- bin/lem generate -kv-cache turboquant:4, real e2b-4bit, median of 3
(before = base commit 8600143, kernels+lem rebuilt from it; after =
this commit; native = no -kv-cache, same build):
single-pass (-context 800, ~730 tok deep): before 116.0 tok/s ->
after 134.9 tok/s (+16.3%); native 172.6 tok/s reference.
2-pass (-context 2200, ~1980 tok deep): before 117.0 tok/s -> after
115.9 tok/s (flat, within run-to-run noise); native 171.6 tok/s.
- go vet ./engine/metal/ clean; gofmt clean on all 6 touched files.
Co-Authored-By: Virgil <virgil@lethean.io>
… pre-checkpoint Establishes exactly what #37 asked: DFlash is a REAL, distinct external drafter family (z-lab/RedHatAI publish real checkpoints on HF, one with 226k downloads) — not a stale backlog entry, not an internal codename. But engine/metal/assistant_dflash*.go was built from the arXiv description before any real checkpoint existed to check it against (its own commit message says so), and checked now against three real checkpoints across the two conventions the wild actually uses, its architecture does not match either — itemised at both the config level (wrong JSON keys/nesting, verifier.name vs the real .name_or_path) and the maths level (gemma4 sandwich-norm+GELU reused verbatim where every real checkpoint is a plain 2-norm qwen3 layer with SiLU; no intra-block self-attention where that IS the actual diffusion mechanism, not a documented-later refinement; numAux conflates fused-layer-count with cross-attention context length; invented dflash.lm_head/d2t tensors no real z-lab checkpoint carries). docs/design-dflash.md (the memo these files still cite) was retired 2026-07-14 once the contract slice + engine forward "landed" — verified against its own assumed shape, never against reality. DFlashEngineProbe has stayed honestly false throughout; this survey explains precisely why it should stay false until the punch list in §6 lands, rather than being flipped against a still-wrong forward. Co-Authored-By: Virgil <virgil@lethean.io>
ParseConfig previously matched only speculators_model_type=="dflash" — the RedHatAI/speculators-library shape — and even there read the wrong verifier key (.name; every real checkpoint found uses .name_or_path). z-lab's own convention (every checkpoint the paper's own lab publishes, z-lab/Qwen3-4B/8B/27B/35B-A3B-DFlash, gemma-4-26B-A4B-it-DFlash, ...) sets no speculators_model_type at all: its marker is architectures:["DFlashDraftModel"] beside a dflash_config object nesting target_layer_ids/mask_token_id. No checkpoint this survey found (z-lab or RedHatAI) was recognised by the old single-convention check. Both conventions set architectures:["DFlashDraftModel"], so that alone is now sufficient to recognise either; field reads try each convention's own path (aux_hidden_state_layer_ids vs dflash_config.target_layer_ids, top-level vs nested mask_token_id, .name_or_path vs .name) with sane fallback. Adds Config.MaskTokenID — real generation bookkeeping (the block's not-yet-drafted positions' starting embedding) every real checkpoint declares and the old struct had no field for at all. New tests use real checkpoint config.json byte content verbatim (z-lab/ Qwen3-4B-DFlash-b16, RedHatAI/NVIDIA-Nemotron-3-Super-120B-A12B- speculator.dflash), cited by source repo — never guessed. All 17 existing tests unaffected (backward-compatible struct addition, no positional literals at any call site). Gates: go build ./... && go vet ./... clean (whole module); gofmt clean; go test ./decode/dflash/... — 30 passed (was 17). Co-Authored-By: Virgil <virgil@lethean.io>
…le-gated ZLabForward is a from-scratch, engine-free (no cgo, no GPU) pure-Go f32 implementation of DFlashDraftModel.forward() — the REAL architecture every published z-lab checkpoint uses, correctly shaped per the #37 survey (docs/design-dflash-survey.md): plain 2-norm qwen3 decoder layers, k_norm on the CONCATENATED context+noise K (not the injected-context-only rows engine/metal assumes), genuine intra-block self-attention joined into the SAME softmax as the cross-attention context (the actual diffusion mechanism, not a deferred refinement), SiLU-gated MLP, fc+hidden_norm context fusion. It borrows no target lm_head/embed_tokens (z-lab's checkpoints carry neither), matching the real tensor set exactly — fc.weight/hidden_norm.weight/layers.N.*/norm.weight, no "model."/"dflash." prefix, no invented reduced-vocab head. Oracle-gated two ways: a synthetic varied-weight Good/Bad/Ugly suite (shape, determinism, non-vacuousness, honest refusals — runs always, no download) and TestZLabForward_RealCheckpoint, env-gated on LTHN_DFLASH_ZLAB_CKPT (the model/quant/awq LEM_AWQ_REFERENCE_DIR pattern — skips cleanly without it) against testdata/zlab_qwen3_4b_oracle.json: exact inputs/expected outputs at 3 depths (after layer 0, after layer 2, final norm) from an independent numpy re-implementation run over z-lab/Qwen3-4B-DFlash-b16's REAL downloaded weights — itself cross-checked against that checkpoint's own modeling_dflash.py executed through transformers (trust_remote_code), not just against my own reading of it. Receipt, run against the real checkpoint (LTHN_DFLASH_ZLAB_CKPT set): after_layer_0: max abs diff 0.079 across 10240 values (hidden std ~120) after_layer_2: max abs diff 0.100 across 10240 values (hidden std ~135) final_norm: max abs diff 0.0023 across 10240 values (hidden std ~2.6) This does not change what serve or engine/metal do (DFlashEngineProbe stays false — see survey §6 for why and the precise punch list); it proves the architecture reading is correct against real weights before any engine-side rewrite, the model/needle package's own "de-risk before an accelerated port" posture. Gates: go build ./... && go vet ./... clean (whole module); gofmt clean; go test ./decode/dflash/... — 30 passed, 1 clean skip without the env var, 30 passed (incl. the real-checkpoint gate) with it set. Co-Authored-By: Virgil <virgil@lethean.io>
…lane/tqperf) Πq/Πᵀy fused into lthn_sdpa_vector_tq (3→1 dispatches per TQ layer per token); masked-word unpack widening in both TQ kernels. Parity band TIGHTENED to ≤0.0020 (two bf16 round-trips eliminated); live gates unchanged (62/64, retrieval). Receipt (E2B-4bit, medians of 3): single- pass 116.0→134.9 tok/s (+16.3%), native 172.6; 2-pass deliberately unfused (per-block rotation replication — the fusion rule inverted) and flat at 115.9, confirming the boundary cost nothing. Co-Authored-By: Virgil <virgil@lethean.io>
Survey verdict: DFlash IS a real external drafter family (z-lab, arXiv 2602.06036 — Qwen3.6-35B drafter at 226k downloads; z-lab even ships gemma-4-26B-A4B-it-DFlash). The existing engine forward was built from the paper BEFORE any checkpoint existed and matches NEITHER real convention (gemma4 sandwich-norm+GELU vs the real 2-norm qwen3+SiLU; missing the intra-block attention that IS the mechanism; invented tensors). DFlashEngineProbe correctly stayed false throughout — the honest decline held. Landed: ParseConfig for BOTH real conventions (+ the .name vs .name_or_path bug), ZLabForward pure-Go f32 oracle- gated against real downloaded weights (final_norm max|d| 0.0023 @ std 2.6), 30 tests, the survey doc with the engine-rewrite punch list. Co-Authored-By: Virgil <virgil@lethean.io>
…reference dumps The E2B capture-vs-oracle divergence (#44: layers 8-14, worst 11 at 0.83) needed a sublayer-resolution instrument to name its first diverging station: - decode_forward_arch.go grows a post-MLP capture point (capturedMLPResHiddens, captureLayerHiddens-gated like its attn sibling) so the serial walk exposes attn_res -> mlp_res -> layer_out per layer. - testdata/e2b_mirror_oracle.py additionally dumps mlp_res.f32 (residual after the MLP, before the PLE gate) — the matching oracle station. - testdata/e2b_mlx_reference.py (new) captures mlx-lm's OWN bf16 per-layer hiddens for the same parity ids into the same dump dir — the bf16 reference implementation the discriminator gates against. - train_e2b_sublayer_probe_test.go (new, env-gated) drills a layer into stations against the oracle dumps: per-layer-input rows, PLE weight fingerprints (transpose-invariant sums + layer scalar), attn/mlp/out station cosines, optional f32 station dumps (E2B_SUBLAYER_DUMP_DIR) for offline delta analysis. Receipts from the instrument (fb0b16 snapshot, 8 parity ids): pli rows >= 0.999991 all 35 layers; PLE gate/proj/norm sums + scalars bit-exact vs the checkpoint; first clean-input break = layer 8 SECOND half (attn 0.999788, mlp 0.999171, out 0.966658), token-resolved to parity token 6. Co-Authored-By: Virgil <virgil@lethean.io>
…e defect; gates go HARD at the correct bar MECHANISM (named): chaotic amplification of correctly-rounded bf16 residual-stream storage through E2B layer 8's gelu cliffs — not a wrong per-layer table, boundary, rope region, or kernel. The proof chain, each step from the #49 sublayer instrument on the real E2B bf16 checkpoint: - Every consumed per-layer parameter is bit-exact (PLE gate/projection/norm sums, layer scalars) and the engine's per-layer-input rows match the oracle >= 0.999991 on all 35 layers. - Every sublayer is arithmetically FAITHFUL: recomputing layer 8's MLP half and PLE half in f64 from the ENGINE'S OWN captured inputs reproduces the engine's outputs to 1-cos <= 3e-6 — and reproduces the ORACLE when fed the oracle's inputs. There is no defective station. - The input difference driving it is storage dust: gemma giant channels (|h|~100, 1 bf16 ULP = 0.5) drift +-2 ULP over layers 0-7 (correctly-rounded random walk; per-station cosines >= 0.99987). Layer 8, parity token 6: the MLP's 38 near-cliff gelu gates amplify 1-cos 2e-4 -> 6e-3 (x30); the PLE gate (branch rms >= residual rms, pli values to 18) amplifies 6e-3 -> 0.30 (x50). Tokens 0-5,7 stay <= 1e-3 through layer 12; token 7 catches 3e-2 from token 6's polluted KV. The pli-dominated giant branches at layers 13/14 (rms 6.5/12) are pli-driven and wash the error back down — the observed "recovery". - DECISIVE: mlx-lm's own bf16 forward (testdata/e2b_mlx_reference.py) diverges from the same f64 oracle with the SAME signature — first bad 8, worst 11 at 0.8316 (engine 0.8342), same recovery, same token-6 concentration — while engine-vs-mlx stays >= 0.9985 at EVERY layer. bf16 forwards form a tight equivalence class (station rounding lands on the same bf16 grid); the f64 trajectory exits that class at the chaos layers. A 0.9999-vs-f64 bar is unreachable for ANY bf16-stationed forward, the ecosystem reference included. GATES (flipped to HARD at the correct bar): - TestForwardCaptureHiddensE2BVsOracle now t.Fatalf's per layer on: engine-vs-mlx < 0.9975 (observed floor 0.99851); engine trailing the mlx ground-truth envelope by > 0.0075 (observed worst gap 0.0045 at layer 8); any pre-chaos layer (mlx >= 0.9999 vs oracle) below 0.9998 (observed >= 0.99993). Green on BOTH routes (ICB worst-oracle 0.8342, serial 0.8385). - TestLoRATrainerE2BSharedKVSFT_Good's B=0 anchor is HARD again at the envelope tripwire (worst-layer >= 0.80; observed 0.8304); live SFT smoke green: loss 18.5051 -> 9.0720 over 5 steps, adapter round-trip 100 tensors, consumers carry no k/v. Quant path verdict: same per-layer tables through the same stepTokenEncode walk (ple[li], lb[li], pliOff) — proven correctly indexed by the bf16 investigation; no defect class exists to port a fix for. Serve path untouched: DecodeForwardArch|HostReference|Parity = 95 passed; no serving code changed. Co-Authored-By: Virgil <virgil@lethean.io>
…proven + hard-gated (lane/fwd49) f64 recomputation from the engine's own captured inputs reproduces the engine to 1-cos <= 3e-6 (every station faithful); mlx-lm's own bf16 forward diverges from the same f64 oracle with the IDENTICAL signature (first bad 8, worst 0.8316 @ 11 vs engine 0.8342) while engine-vs-mlx >= 0.9985 all layers — bf16 forwards are an equivalence class and the 0.9999-vs-f64 bar was unreachable for ANY of them. Mechanism: +-2 ULP giant-channel storage dust -> layer-8 gelu cliff (~30x) -> PLE gate (~50x) on one token, washed back by the giant PLE branches at 13/14. Discriminator now Fatalf at the PROVEN bar (vs-mlx >= 0.9975, envelope, pre-chaos >= 0.9998, both routes); E2B SFT anchor HARD again (worst 0.8304 >= 0.80; loss 18.5051 -> 9.0720). Quant path: same tables, correctly indexed, no defect class to port. Co-Authored-By: Virgil <virgil@lethean.io>
…-f32
DOTS-OCR moves from "recognised, refuses" to working library-level OCR, per
the whisper template: real checkpoint (rednote-hilab/dots.ocr, verified via
snapshot_download against the recogniser's own naming), real tensor map
(never guessed — every name/shape/bias-presence confirmed against the actual
643-tensor safetensors index), host-f32 vision tower + decoder forward, and a
public OCR(imageBytes, prompt) entry point.
Architecture (confirmed from the shipped custom_code, not assumed):
- Text decoder is an UNMODIFIED Qwen2ForCausalLM (GQA, 1D RoPE theta=1e6,
SwiGLU, untied lm_head — modeling_dots_ocr.py subclasses Qwen2ForCausalLM
directly and only overrides embedding preparation).
- Vision tower is NaViT-style (modeling_dots_vision.py): a Conv2d patch embed
that folds exactly into a dense [1536,588] projection (verified 0.0 max-abs
-diff against the real nn.Conv2d module), 2D rotary (h/w split halves),
full (non-causal, single-image) attention, SwiGLU, PatchMerger.
- Image preprocessing replicates transformers' Qwen2VLImageProcessorPil
(smart_resize + patchify): dimension maths and normalise/patchify are
bit-exact vs the real processor; pixel resampling for non-aligned images
is a named approximation (bilinear vs the reference's PIL bicubic) —
invisible to the committed fixture, which is pre-aligned to the 28px grid.
- Prompt construction was cross-checked empirically against the real
chat_template + processor, not derived from reading the Jinja alone — a
first attempt got the <|user|>/<|endofuser|> wrapper wrong for list-content
messages; the golden catches it.
Reference-run method: torch 2.11 + transformers 5.5.4 (trust_remote_code),
float32 CPU, in the venv named in the task brief. Found and worked around a
real from_pretrained loading defect in this environment: DotsVisionTransformer's
VisionRotaryEmbedding.inv_freq (a non-persistent, __init__-computed buffer)
comes out all-zero after AutoModelForCausalLM.from_pretrained, while a freshly
constructed instance computes it correctly — confirmed by direct comparison.
Unpatched, vision attention silently loses all positional information; this
is very likely why an early capture produced degenerate repeated-bbox output.
Fixed in the capture script by rebuilding the submodule and copying its
correct buffer over the loaded one before capturing any vision golden.
Goldens (testdata/, captured against the real checkpoint): smart_resize
dimension table, image preprocessing (sampled patch rows vs the real
Qwen2VLImageProcessorPil on the committed fixture.png), a real-weights vision
block/tower pass on a synthetic grid, a real-weights decoder layer/stack pass
on a short prompt, prompt-construction cross-check, and one full E2E: the
committed fixture.png ("Lethean OCR 2026", PIL-generated, testdata/gen_fixture.py)
through the real checkpoint's own forward() (model.generate() itself throws
inside this checkpoint's prepare_inputs_for_generation under this transformers
version — a real custom_code incompatibility, not this port's bug; worked
around with a hand-rolled greedy loop calling the same real forward()).
Gates: go build/vet clean on the touched packages AND the whole go/ and
examples/ modules; gofmt clean; 49 hermetic tests + 3 Examples green
(smart_resize, patchify, prompt construction, KV-cache batching parity,
scatter, config/weights error paths — all package-private, white-box). 7
env-gated live tests (DOTS_OCR_DIR) against the real checkpoint ALL PASS,
including the E2E OCR() call reproducing the reference's exact greedy
generation verbatim — timings load-caveated (this box, sibling lanes
resident): Load ~2s, vision-block checks ~2-7s each, decoder full-stack
~10s, full E2E OCR ~548s (245-token prefill + ~40 generated tokens, pure-Go
scalar float64 accumulation, no BLAS — device fusion is explicitly out of
scope for this lane).
Boundaries named plainly: bilinear (not bicubic) resampling for images
needing an actual resize; single static image per call (no video/multi-
frame batching — matches the OCR() signature); temporal_patch_size and
num_channels other than the checkpoint's own (1, 3) refuse rather than
guess; register.go's Composed/Arch refusal is untouched — this ships as a
separate own-loader entry point, exactly like whisper's, and does not touch
go/engine/**, serving/, model/composed/, or cli/.
Co-Authored-By: Virgil <virgil@lethean.io>
The 26B-A4B pair decoded 50 tok/s against 138 plain AT 64-71% acceptance — the verify forward was flat ~43-55ms for K=3 AND K=5 (a plain step: 7.2ms). LTHN_GPU_TRACE=host indicted the seam, not the GPU: every verify round paid syncKV 5-12ms + reloadKV 25-42ms, BOTH growing with position — a full-prefix paged->linear->paged round-trip (with q8 recode both ways, the sync-side dequant single-threaded) to move K=5 rows. The winning E2B pair pays 0.0ms. Fix: copy only the rows that changed. syncLinearKVFromDevicePaged copies [watermark, basePos) direct page->lb (no more full linearSnapshot + clear per round); reloadDevicePagedKVFromLinear pushes only the batch's landed rows [pool high-water, basePos+K) back via slot(). Ring caches get an absolute mirror watermark (linearSyncedAbs) — slot() overwrites and truncate() lower it; loadLinearSnapshot zeroes it (state-restore provenance unknown). The full paths remain as in-code fallback for undeclared geometry and behind LTHN_KV_SYNC_DELTA=0 (repro anchor). q8 pages now code each landed row ONCE (the serial lane's coding) instead of re-quantising the whole prefix each round. Receipts (26B-A4B qat-4bit + official assistant, temp 0, box under 3-lane load — ratios load-insensitive, absolute ms caveated): pair decode 50.1 -> 106.4 tok/s (128 tok, 67% acceptance unchanged) syncKV 5-12ms/round -> 0.0ms reloadKV 25-42ms/round (position-growing) -> 2.3-4.4ms flat (K rows) verify fwd 43-55ms -> 14-21ms Scoped gates: paged-KV suite 27 green (9 new one-per-symbol tests); MTP|Mtp|Speculative|Reforge 152 green; Verify|MoE|Moe|Assistant 344 green, 1 pre-existing fail (TestMoEBlockBF16AllocationBudget allocs=5>4) which fails identically at clean HEAD 5353d20 in the untouched main tree. Co-Authored-By: Virgil <virgil@lethean.io>
Stage the K verify rows' attention as concurrent waves on the round's single command buffer instead of K serial encAttnHalfKVPagedInputAt calls: W1 RMS-norms, W2 q/k/v projections, W3 ropes + value-norms, then landing + SDPA, W7 output projections, W8 residuals, with memoryBarrierObject only at true stage edges. Global KV layers land all K rows' pages then scan barrier-free (causal lens baked per row); ring/sliding layers keep the per-row land->scan chain so later landings still evict inside earlier rows' windows in eviction order. The causality trap en route: buildSDPAPagedDecodePlan borrows the cache's shared state() scratch slices, so deferring K emissions past K state() calls aliased every plan onto the last row's lens. Per-row deep copies of the seven borrowed slices fix it; the RowsDriver byte-parity gate caught the miss. 26B-A4B pair receipt: 93.2 -> 104.3 tok/s (400 tok / 3.826s), 60% acceptance (347/575) over 115 verify forwards, output byte-identical to the pre-campaign pair. Cumulative for the campaign: 15.6 -> 104.3 (6.7x), router per-row reduction untouched throughout. Full metal suite 2796 green. Co-Authored-By: Virgil <virgil@lethean.io>
…hed canonical head, ring waves below the wrap point Three submission-shape levers on the #53 chained round, each output-invariant by construction and receipted on the canonical 26B-A4B pair (greedy, 400 tok, counting prompt), text byte-identical at every step: - Chunked commits (rowsChainCommitStride): the round previously encoded all layers, THEN ran — the GPU sat idle behind ~6ms of CPU encode. Committing each layer's buffer as soon as it is encoded (stride 1 won the sweep: 135.9/135.3/133.7/132.5/130.0 tok/s at 1/2/4/6/8) lets execution chase the encoder. Cross-buffer ordering is the device timeline's: every buffer is default hazard-tracked and consecutive chunks always depend through the rotating slabs. 104.3 -> 116.0. - Batched canonical head (greedyRowsCanonicalInPool): the verify rows all resolve to greedyInPool's upload + encodeGreedyAt chain, K commit+wait round-trips deep. One command buffer, K unchanged per-row chains on their own scratches, one wait. This batches the CANONICAL argmax tier by submission shape only — not the qmm_t rows-head tier, which stays behind LTHN_MTP_ROWS_HEAD (#55). 116.0 -> 117.7. - Ring wave below the wrap point: slot() is pos%maxSize, the identity while the round's landings stay under maxSize — no eviction is possible, so sliding layers take the global land-all-then-scan wave shape (virgin-slot argument) instead of K serial land->scan chains. Rounds that CAN wrap keep the serial chain (eviction order is the byte contract). 25 of the 26B's 30 layers are sliding. 117.7 -> 130.0. New parity pins: TestMTPRowsDriverVerifyMatchesRowMajorSliding_Good runs the driver-vs-row-major byte compare on an all-sliding fixture at BOTH regimes (window 16 = wave, window 4 = serial wrap). Full metal suite 2799 green; interleaved 3x receipt below with the block-default commit. Co-Authored-By: Virgil <virgil@lethean.io>
A quant-MoE target's verify round carries a higher fixed cost (router + expert gathers + the K-row head amortise per round), so a longer draft block pays there and only there. The 26B-A4B block sweep read 127.4 / 135.6 / 135.9 / 137.4 / 136.7 / 129.6 tok/s at blocks 3..8 (knee at 6); the SAME sweep on E2B (dense-PLE) regressed 96.6 -> 88.5 at 6. LoadSpeculativePair now resolves a deferred block (<=0) to 6 when the attached target arch HasMoE(), 5 otherwise; an explicit -draft-block always wins. The generate CLI previously pre-substituted the shared constant 5 before the loader ever saw the flag — it now passes 0 through, same shape as serve's resolver, so the loader owns the default in both lanes. Boot notices print "engine-default block" instead of pre-claiming a number the notice cannot know. THE GAUGE (#53, "mtp can't be slower than without"): interleaved 3x on the canonical 26B receipt, this build, defaults — plain 132.6 / 133.3 / 132.5 vs pair 137.7 / 137.1 / 137.4 Pair beats plain by ~3.6% at defaults, output byte-identical, 72% acceptance over 83 verify forwards. Campaign cumulative: 15.6 -> 137.4 (8.8x). E2B default lane unchanged (115 forwards = block 5). Co-Authored-By: Virgil <virgil@lethean.io>
…s + §7d The missing comparator from §7c's honest conclusion now exists: the checkpoint's OWN spec_generate (torch/MPS, method source re-exec'd with only the return widened) reads 17.10% accept / 3.565 tokens/round on the survey prompts vs our 7.8% / 2.087 — the pairing is healthy, the gap was real, and the hunt is now CONCLUDED, not open: fused context 0.3%, layer-0 attention 0.6%, all five layers 0.7-2.5% at every position on torch-exact inputs, target embedding row byte-exact — no structural fault in the engine forward. The gap's mechanism is numeric-regime tie correlation: the drafter's argmaxes are knife-edge (the same forward flips fib's match depth 9 -> 1 on a ±0.25% input change) and the checkpoint's ties were shaped by torch-bf16 rounding. Full evidence and the one honest fix lane (a torch-bf16-equivalent forward mode) in docs/design-dflash-forward.md §7d. New env-gated instruments (skip without the real checkpoints): assistant_dflash_zlab_round1_test.go — reference-frame round-1 proposal log; assistant_dflash_zlab_xfeed_test.go — torch-exact context feed with per-stage isolation (fused ctx, layer bisect, l0 attention, embedding byte-parity). Both carry the embedID shared-scratch pin the first bisect run paid for learning about. DFlashEngineProbe stays false; whether the bf16-fidelity lane is worth building for a non-gemma4 side drafter is a prioritisation call. Co-Authored-By: Virgil <virgil@lethean.io>
…istic lane per stream The batched small-K fold (qmm token-identity tier, the E2B 206-vs-131 lane) becomes the DEFAULT verify tier for every lane, greedy included (Snider 2026-07-21). #55 fenced it from greedy because composed with the #299 wall-clock re-engagement machinery the lane switching made timing observable as flipped near-tie tokens (the q8race bistability). The rearm keeps determinism the other way round: when the greedy stream folds, GenerateFromSessionEach pins the wall-clock machinery OFF (wallReengage) — drafting stays engaged, the only bail is the count-based low-accept patience finishing the request plain permanently, and the emitted stream is a pure function of the input again. LTHN_MTP_VERIFY_FOLD=0 restores the byte-exact per-row lane everywhere (the parity-forensics anchor); LTHN_MTP_REENGAGE=0 keeps its pre-#299 meaning on that lane. Quant-MoE targets are untouched (their dense fold declines structurally; the rows driver continues to serve them). Receipts (canonical greedy 400-tok counting prompt, this box): - E2B qat + bf16 assistant: pair 96.6 -> 218.4 tok/s (plain 219.8 — a tie at 58% acceptance; block sweep 4/6/8: 216.1/219.1/184.7). - E4B qat + qat assistant: pair 149.8 (plain 149.0); + bf16 assistant 151.6. - 26B-A4B: unaffected (rows-driver lane), pair 137.4 vs plain 132.6 stands. Test contract updates: TestMtpVerifyFoldArmed_* pin the rearmed routing; the rows-driver parity pins and the two wall-clock re-engage scenario tests pin the LTHN_MTP_VERIFY_FOLD=0 forensics lane their subjects now live on. Full metal suite 2799 green with the rearm. Co-Authored-By: Virgil <virgil@lethean.io>
…lean E4B The rearmed fold exposed a pre-existing corruption (reproduces pre-rearm under LTHN_MTP_VERIFY_FOLD=1): the uniform-4-bit lean E4B conversion (mlx-community/gemma-4-e4b-it-4bit) produces all-NaN verify rows on the recorded whole-stack verify ICB, surfacing as an invalid direct-argmax token. Banked bisect (task #71): target-side not drafter-side (cross-pair receipts); verifyStackICBDisabledForTest clears it while the verify-tail lever does not; qmm_t is byte-clean at every e4b dim at bits=4 in isolation; a bare non-ICB session serves the same verify clean; plan routes are identical at bits 4 and 8. Workaround: LTHN_VERIFY_STACK_ICB=0 — the plain-4bit E4B pair then runs clean at 173.4 tok/s, BEATING the qat pair's 151.6. Stack-replay value on healthy checkpoints: qat E4B +3%, E2B nil — default stays on pending the root cause. Instruments: TestBatchedDenseFoldE4BPlain4BitNaN_Ugly (env-gated live-stage bisect over the real pair lane, self-reporting which lever clears the failure), e4b dim cases in TestQMMTAt31BDims, and NaN/stats forensics on greedyRowsCanonicalInPool's invalid-argmax error path (mirroring argmaxInvalidDiag one tier up). Co-Authored-By: Virgil <virgil@lethean.io>
TestVerifyStackICBReplay_QMMTParity_Good: the recorder-recorded qmm_t replayed via executeInto is byte-clean against the per-row qmv truth at every live E-series MLP dim at bits 4 AND 8 (synthetic weights, no model, runs in-suite) — proving the single-command ICB replay path and narrowing #71's all-NaN corruption to the multi-command stack composition. The record-arm trace line (LTHN_VERIFY_STACK_DEBUG) logs K + basePos at recorder arming, which is what falsified the rewind/reset and carry/width theories: the failing replay is a plain forward-sequence first replay, any K, E4B interiors only (lean E2B replays clean eight deep). Full bisect state in task #71. Co-Authored-By: Virgil <virgil@lethean.io>
…ne and per-pass mutation exonerated The E4B first-replay NaN hunt, session 3. Everything mutable per pass is now falsified with live receipts: identity replay (same PLE slab via the grow-only LTHN_VERIFY_STACK_PLE_PAD probe, no rebind rewrites via LTHN_VERIFY_STACK_NO_REBIND, same basePos) still NaNs; the whole q8 lane is out (LTHN_KV_Q8_ICB=0 bf16 recording still NaNs — the q8 scale corruption the census shows at exactly the landing rows is a symptom: the store quantises an already-poisoned stage); no single op class is the poison (honest LTHN_VERIFY_STACK_SKIP_OP ladder — a skipped op's markRebindLast used to fail the whole recording vacuously, lastSkipped now guards the marks); scale metrics are out (skipped streams at 612 cmds / 40 rebinds / 0 dyn — all below E2B's clean 678/89/33 — still NaN). Instruments (all env-gated, default-off): - LTHN_VERIFY_STACK_DEBUG: per-op-class histogram + layer-1 op sequence + per-layer command spans in the record-finish trace (layer-1 streams are op-identical E4B vs E2B; only dims differ). - LTHN_VERIFY_STACK_ROWHASH=1: pass-end census — fold slabs, readRows, q8 K-scale planes (names the corruption channel and the landing offsets). - LTHN_VERIFY_STACK_REPLAY_LAYERS=N / _REPLAY_CMDS=M: prefix replay bisect (recorded interior through layer N / command M, live continuation). N=1 NaNs on E4B and is clean on E2B. - LTHN_VERIFY_STACK_SKIP_OP=<class>: omit one op class from the recording. - LTHN_VERIFY_STACK_NO_REBIND / _PLE_PAD: identity-replay probes. Green micros pinning what is NOT broken: the 7-op MLP chain (rms, gate, up-with-overlapNext, gelu, down, rms, add) at both models' dims, and the kv-q8 store byte-parity at kvDim 512/1024/256 landing at basePos 26. Full state + next moves in task #71. Co-Authored-By: Virgil <virgil@lethean.io>
…ags; poison map banked Bottom-up map from the clean adds-only base: proj and rms@2560 are named poison classes (each NaNs the base alone; all seven other classes are clean alone); skip=proj,rms on the full stream still fails, so an interaction poison remains. The decisive repro: the adds+rms2560 stream cut to TWO interior layers (12 fully-barriered rms/add commands) NaNs on E4B and is clean at one layer — and E2B replays the identical stream clean at five. Layer 2 is the first in-place ping-pong layer. Full state in task #71. Co-Authored-By: Virgil <virgil@lethean.io>
…dd micro; shard offsets >2^32 sighted The captureBind dump on the minimal poisoned stream exposed the real seven-command layer topology (the PLE epilogue's fused rms_residual writes the rows a second time in place) and — the lead — norm weights binding the ~5GB mmap shard at offsets beyond 2^32 on E4B (4.94GB, 5.11GB), while E2B's whole shard sits under 4.3GB. An ICB-side 32-bit buffer-offset limit would explain the model split and the proj/rms poison classes exactly; the committed two-layer micro (green at small offsets) gains >2^32-offset cases next to give the verdict. Own-encoder replay falsified. Full state in #71. Co-Authored-By: Virgil <virgil@lethean.io>
…ts >= 2^32; recorder declines such binds The E4B first-replay NaN, hunted across three sessions, lands on a Metal platform limit: an indirect compute command's setKernelBuffer offset is not faithfully encoded at or above 2^32 bytes. The live encoder path is exact at any offset; the replayed ICB bind reads the wrong bytes. E4B's ~5GB lean shard binds its interior norm and projection weights at 4.3-5.1GB offsets — every whole-stack verify replay corrupted on first execution (records were clean because record-along passes are served by the live encodes). E2B's shard sits entirely under 2^32, which is the whole model split. Proof: TestVerifyStackICBReplay_RMSAddTwoLayerParity_Good/e4b-2560-2layer-shard4g — identical two-layer rms/add stream, weights at 4.0001GiB offsets, replay diverged from live at cos 0.75 with every smaller-offset variant byte-exact. Fix: verifyStackRecorder.captureBind fails the recording on any bind offset >= 2^32 — the lane auto-declines on >4GiB-shard models and the live fold serves. E4B lean pair now runs 168 tok/s clean with NO kill-switch env (was: hard invalid-token abort without LTHN_VERIFY_STACK_ICB=0); E2B keeps its replays (210.8 tok/s, unchanged). Follow-up (new task): re-enable stack replay on >4GiB shards (re-materialise small norm weights / page-aligned windowed sub-buffers over the mmap for projections) and audit the other ICB lanes (arch decode, verify-tail, fused chain) for the same offset class. Co-Authored-By: Virgil <virgil@lethean.io>
…— rebase high binds onto no-copy windows A bind at offset >= 2^32 now rebases onto a memoised page-aligned no-copy window over the same memory at the containing 2GiB segment base — the recorded offset drops under 2^31, the bytes are identical, nothing is copied. The shard mmap's exact-file-size tail is page-floored (its final partial page is mapped regardless); a bind STARTING inside that fragment declines the recording, as does an unbuildable window (captureBind's >=2^32 guard stays as the backstop). E4B lean pair: replays re-engage at every position, 175.8 tok/s (was 168.4 declining, 173.4 under the old env kill), output BYTE-IDENTICAL to the live fold (LTHN_VERIFY_STACK_ICB=0 A/B). Pins: shard4g micro now asserts full record+replay parity through the windows; TestVsRebaseHighBind_Ugly covers the segment reuse, tail-fragment decline, and out-of-range edges. Co-Authored-By: Virgil <virgil@lethean.io>
The #71 class closed across all four recording lanes: verify-tail declines via fail() after an unrebasable window; the fused chain and the arch lane rebase through the shared no-copy windows (their weights are per-tensor buffers today, so the branch is future-proofing) and trace loudly if a window cannot build — neither has a decline channel at the sink. Co-Authored-By: Virgil <virgil@lethean.io>
…in the batched verify interleave, not the fold The 12B pair (gemma-4-12B-it-4bit + bf16 assistant) diverges from the per-row byte-exact verify lane at the first boundary and collapses to 25% acceptance. The env-gated bisect proves the per-row reference sound and every fold-stage lever innocent: multiq/rope/epilogue/mlp-fold all still diverge — INCLUDING mlp-fold-off, i.e. the whole fold declined and the batched per-row interleave serving — and the divergence point shifts (27->17) under any lever, so the model sits on knife-edge argmax ties that any accumulation-order delta flips. E2B/E4B/26B hold byte parity through the same machinery; a 12B-specific op in the interleave is not order-identical to sequential. Hunt continues in #73. Co-Authored-By: Virgil <virgil@lethean.io>
…ings LoadDir/LoadDirMmap knew two layouts: an index.json-mapped shard set and a single model.safetensors. A sharded checkpoint WITHOUT an index — standard model-NNNNN-of-NNNNN.safetensors or the model.safetensors-NNNNN-of-NNNNN .safetensors spelling Qwen3.5 snapshots ship — was rejected outright. A new scan route (isShardName/scanShards) now finds shard-named files in either spelling when neither the index nor the single file exists, merged in sorted order; a stray non-shard tensor file (adapter.safetensors) never matches, and a tensor claimed by two shards refuses rather than silently picking a winner. Receipts: TestSharded_LoadDir_ShardNamingVariants + TestSharded_LoadDirMmap_ShardNamingVariants table the three naming forms; isShardName/scanShards carry their own Good/Bad tables; the extended TestSharded_LoadDir_Bad proves the stray-only and duplicate-tensor refusals. Co-Authored-By: Virgil <virgil@lethean.io>
…snapshots assemble Qwen/Qwen3.5-0.8B and -2B HF snapshots ship the current transformers multimodal nesting — model.language_model.* text tensors beside model.visual.* and mtp.* — but NormalizeWrapperNames only knew the classic language_model.* wrapper prefix, so no bare model.* alias was ever created and every load died at "model.Assemble: model.embed_tokens absent" (the weights themselves were found fine: the snapshots' index.json maps their model.safetensors-00001-of-00001.safetensors shard, which LoadDirMmap already follows by name). The new wrapperStripped helper recognises both layouts — language_model.X -> X and model.language_model.X -> model.X — keeping the same-map second-pass return and the no-clobber rule intact, so Assemble's bare lookups, InferFromWeights' probes and assembleGatedDelta's linear_attn reads all resolve; model.visual.*/mtp.* pass through untouched. engine/hip's DenseWeightNameCandidates already lists this prefix as a standard alias — this brings the factory route level. Receipts: TestWrapperNames_WrapperStripped_Good/_Bad table both prefixes and the non-text siblings; TestWrapperNames_NormalizeWrapperNames_ NestedWrapperLayout proves the aliasing + the #60 writeback contract on the new layout; both snapshots' weight maps verified to carry exactly the model.language_model/model.visual/mtp prefixes, no lm_head (tied). Co-Authored-By: Virgil <virgil@lethean.io>
…hards) Qwen3.5 HF snapshots load: the current transformers nesting (model.language_model.*) aliases through NormalizeWrapperNames, and index-less shard scans accept all three safetensors naming forms. Co-Authored-By: Virgil <virgil@lethean.io>
The hunt's verdict, receipts in the task: no bug. The greedy divergence is the documented fold token-identity contract surfacing on 12B's knife-edge argmax ties (every stage lever still diverges and the divergence point moves with any reordering — the tie signature); the drafter is mid-pack on prose (25% vs E2B's 10%); no regression exists (the July-10 era measured pair 73.4 vs plain 73.3 — the +28% reference never reproduced); and the family truth is that MTP acceptance is predictability-dependent everywhere, with the low-accept bail correctly parking pairs at plain speed on prose. 12B pair stays parity-with-plain by verify economics (48-layer forward vs mid-pack acceptance); the stack replay already serves it post-#72. The test now logs the tie-tier behaviour instead of asserting a parity the contract never promised. Co-Authored-By: Virgil <virgil@lethean.io>
Snider
added a commit
that referenced
this pull request
Jul 22, 2026
design-rocm.md #11. Verdict: keep the two parsers intentionally separate, cross-referenced — not unify. They answer different questions: quantfmt.QuantInfo detects HOW AN EXTERNAL TOOL quantised a checkpoint (the HuggingFace quantization_config key: gptq/awq/ compressed-tensors/fp8/bitsandbytes/quark provenance), which no in-tree engine load path currently reads. model.QuantConfig is THIS ENGINE'S OWN runtime dequant instructions (the MLX-native quantization block), read directly by the assembler via For(name). Different config keys, different shapes, different consumers, zero shared code today — merging them would conflate an import-time detection concern with an execution-time one for no benefit. Both types now carry a doc-comment pointing at the other, and design-rocm.md §B.3 records the resolution inline, so a future reader lands on the right parser instead of building a third one for a third format. Co-Authored-By: Virgil <virgil@lethean.io>
docs/ now holds only documentation — the manual (architecture, backends, build, cmd-lem, development, gui, models, interfaces, types, history, index), the API-compat references (anthropic/ inference/ ollama/ openai/), the state-subsystem contracts (state/), examples/, and the AX RFC. The campaign handover, dated audits, the executed purge plan, todo, perf notes, ten design drafts, receipts/ and superpowers/ move to the orchestrating agent's home — working state, not user documentation. Co-Authored-By: Virgil <virgil@lethean.io>
Co-Authored-By: Virgil <virgil@lethean.io>
First consumer of the freshly released go-html go/v0.11.0 (ctml parser, terminal renderer, box map, teabox). Import + parse probed green; the tab-bar slice lands next per the migration design. Co-Authored-By: Virgil <virgil@lethean.io>
The panel bar is the first screen through the .ctml migration: tabs.ctml (go:embed) parsed by ctml.Parse and rendered by html.RenderTermBoxes, replacing the hand-composed lipgloss strip (intersperse deleted, same visual: LEM brand + per-panel markers, palette colours via TermTheme.Classes from style.go). - Labels bind through ctml.Bindings + <each>, re-bound per change; the active tab rides its own sequence (tabsBefore/tabsActive/tabsAfter) because .ctml class attributes are static strings. - Mouse added: tea.WithMouseCellMotion() on both programs, tea.MouseMsg in Update -> onMouse -> teabox.Resolve against the render's box map; clicks switch panels through the same selectPanel path as Tab/S-Tab (Models first-visit discovery included). Wheel keeps its transcript route; boot/overlay gates mirror the keyboard. - go-html boxes block-level elements only, so per-tab boxes are derived from the ANSI-stripped render (mergePanelTabBoxes) and merged into the same BoxMap; teabox's smallest-box rule prefers tab over strip. - README documents the idiom (markup + Bindings + theme + boxes) for the next slices. Gates (cli/): go vet clean; go build clean; go test -run 'PanelBar|OnMouse|SelectPanel|WorkspaceFrame|ChooseLayout|PanelID' 21 passed; -run 'AppUpdateTransitions|RunWithWorkspace' 5 passed. Co-Authored-By: Virgil <virgil@lethean.io>
…abs) The proof-of-seams slice: tabs.ctml parsed and rendered through go-html v0.11.0's terminal renderer with the palette mapped onto TermTheme classes, per-tab mouse switching resolved through the box map + teabox (mouse enabled for the first time), old lipgloss composition deleted. 374 tui tests green. The lane's go-html friction report (inline boxes, row-scoped each styling, interpolation, whitespace) feeds upstream before slice 2. Co-Authored-By: Virgil <virgil@lethean.io>
…anged The #51 rewrite left main (the squash-merge release line) and dev sharing only the ancient root, so the dev->main PR conflicts structurally. This ours-strategy merge makes main an ancestor of dev with dev's tree byte-for- byte unchanged; the promotion PR becomes a fast-forward and every future dev->main merge is clean. Co-Authored-By: Virgil <virgil@lethean.io>
|
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
No description provided.